{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Recorder import Recorder\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "from TempArgs import args\n",
    "\n",
    "args = args()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from utils import Drawing_extension_time\n",
    "\n",
    "# Drawing_extension_time(args, times=[11, 12])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from utils import Drawing_fisher_muter\n",
    "\n",
    "# bx = Drawing_fisher_muter(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bx_figure = bx.get_figure()\n",
    "# path = 'record/Cifar10/metrics/result'\n",
    "# save_path = os.path.join(path, '{}_{}_{}.pdf'.format('extension_fmuter', args.adv_type, args.isBatchRemove))\n",
    "# bx_figure.savefig(save_path, dpi=400)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from utils import Time_summary\n",
    "\n",
    "# Time_summary(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from utils import Drawing_summary\n",
    "\n",
    "# ax = Drawing_summary(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# perturbed_acc_gap\n",
    "# clean_acc_gap\n",
    "# distance\n",
    "\n",
    "# ax_figure = ax.get_figure()\n",
    "# path = 'record/Cifar10/metrics/result'\n",
    "# save_path = os.path.join(path, '{}_{}_{}.pdf'.format('clean_acc_gap', args.adv_type, args.isBatchRemove))\n",
    "# ax_figure.savefig(save_path, dpi=400)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(args.times)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder = Recorder(args)\n",
    "# recorder1 = Recorder(args)\n",
    "# recorder2 = Recorder(args)\n",
    "Recorder3 = Recorder(args)\n",
    "\n",
    "# recorder.load()\n",
    "# recorder1.load(times=1)\n",
    "# recorder2.load(times=2)\n",
    "time_method_list_1 = ['SISA_shards5_{}'.format(i*500) for i in range(1, 7)]\n",
    "time_method_list_2 = ['SISA_shards10_{}'.format(i*500) for i in range(1, 7)]\n",
    "time_method_list_3 = ['SISA_shards20_{}'.format(i*500) for i in range(1, 7)]\n",
    "\n",
    "time_method_list_4 = ['SISA-DK_shards5_{}'.format(i*500) for i in range(1, 7)]\n",
    "time_method_list_5 = ['SISA-DK_shards10_{}'.format(i*500) for i in range(1, 7)]\n",
    "time_method_list_6 = ['SISA-DK_shards20_{}'.format(i*500) for i in range(1, 7)]\n",
    "\n",
    "other1 = ['MUter'+'{}'.format(i*500) for i in range(1, 7)]\n",
    "other2 = ['FMUter'+'{}'.format(i*500) for i in range(1, 7)]\n",
    "\n",
    "arr = np.concatenate((time_method_list_1, \n",
    "                    time_method_list_2,\n",
    "                    time_method_list_3,\n",
    "                    time_method_list_4,\n",
    "                    time_method_list_5,\n",
    "                    time_method_list_6,\n",
    "                    other1,\n",
    "                    other2,\n",
    "                    ))\n",
    "\n",
    "Recorder3.load(time_method_list=arr, times=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'PGD_2_SISA_shards5_500': array([5596.5823982]),\n",
       " 'PGD_2_SISA_shards5_1000': array([5312.6589379]),\n",
       " 'PGD_2_SISA_shards5_1500': array([5271.5738232]),\n",
       " 'PGD_2_SISA_shards5_2000': array([5673.0030508]),\n",
       " 'PGD_2_SISA_shards5_2500': array([5473.95992947]),\n",
       " 'PGD_2_SISA_shards5_3000': array([5420.08868313]),\n",
       " 'PGD_2_SISA_shards10_500': array([2895.70935678]),\n",
       " 'PGD_2_SISA_shards10_1000': array([2863.80206442]),\n",
       " 'PGD_2_SISA_shards20_500': array([1516.27171969]),\n",
       " 'PGD_2_SISA_shards20_1000': array([1485.33334947]),\n",
       " 'PGD_2_SISA_shards20_1500': array([1467.83913565]),\n",
       " 'PGD_2_SISA_shards20_2000': array([1474.41754103]),\n",
       " 'PGD_2_SISA_shards20_2500': array([1454.6857872]),\n",
       " 'PGD_2_SISA_shards20_3000': array([1431.70720816]),\n",
       " 'PGD_2_SISA-DK_shards5_500': array([1067.01644135]),\n",
       " 'PGD_2_SISA-DK_shards5_1000': array([1001.82748628]),\n",
       " 'PGD_2_SISA-DK_shards5_1500': array([955.18957734]),\n",
       " 'PGD_2_SISA-DK_shards5_2000': array([927.39186931]),\n",
       " 'PGD_2_SISA-DK_shards5_2500': array([1993.3131392]),\n",
       " 'PGD_2_SISA-DK_shards5_3000': array([1930.15677738]),\n",
       " 'PGD_2_SISA-DK_shards10_500': array([254.67867756]),\n",
       " 'PGD_2_SISA-DK_shards10_1000': array([235.73376942]),\n",
       " 'PGD_2_SISA-DK_shards10_1500': array([493.29250813]),\n",
       " 'PGD_2_SISA-DK_shards10_2000': array([469.99109983]),\n",
       " 'PGD_2_SISA-DK_shards10_2500': array([722.06005526]),\n",
       " 'PGD_2_SISA-DK_shards10_3000': array([996.96328044]),\n",
       " 'PGD_2_SISA-DK_shards20_500': array([136.51855898]),\n",
       " 'PGD_2_SISA-DK_shards20_1000': array([121.78060102]),\n",
       " 'PGD_2_SISA-DK_shards20_1500': array([182.66396666]),\n",
       " 'PGD_2_SISA-DK_shards20_2000': array([244.43690896]),\n",
       " 'PGD_2_SISA-DK_shards20_2500': array([305.53882527]),\n",
       " 'PGD_2_SISA-DK_shards20_3000': array([442.44371653]),\n",
       " 'PGD_2_MUter500': array([14.45730948]),\n",
       " 'PGD_2_MUter1000': array([22.90896964]),\n",
       " 'PGD_2_MUter1500': array([31.63827705]),\n",
       " 'PGD_2_MUter2000': array([44.19551802]),\n",
       " 'PGD_2_MUter2500': array([49.96204948]),\n",
       " 'PGD_2_MUter3000': array([60.46986341]),\n",
       " 'PGD_2_FMUter500': array([10.1243825]),\n",
       " 'PGD_2_FMUter1000': array([18.96062684]),\n",
       " 'PGD_2_FMUter1500': array([29.46049809]),\n",
       " 'PGD_2_FMUter2000': array([38.2981863]),\n",
       " 'PGD_2_FMUter2500': array([42.27332997]),\n",
       " 'PGD_2_FMUter3000': array([67.19359326])}"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Recorder3.time_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "recorder4 = Recorder(args)\n",
    "recorder4.load(time_method_list=arr, times=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'PGD_2_SISA_shards5_500': array([5759.01615191]),\n",
       " 'PGD_2_SISA_shards5_1000': array([5483.9583732]),\n",
       " 'PGD_2_SISA_shards5_1500': array([5471.75695221]),\n",
       " 'PGD_2_SISA_shards5_2000': array([5218.5972894]),\n",
       " 'PGD_2_SISA_shards5_2500': array([5114.8429654]),\n",
       " 'PGD_2_SISA_shards5_3000': array([5251.54338912]),\n",
       " 'PGD_2_SISA-DK_shards5_500': array([1043.543534]),\n",
       " 'PGD_2_SISA-DK_shards5_1000': array([1021.53465732]),\n",
       " 'PGD_2_SISA-DK_shards5_1500': array([1204.52522]),\n",
       " 'PGD_2_SISA-DK_shards5_2000': array([1102.5353453]),\n",
       " 'PGD_2_SISA-DK_shards5_2500': array([1945.64534234]),\n",
       " 'PGD_2_SISA-DK_shards5_3000': array([1966.64563321]),\n",
       " 'PGD_2_SISA-DK_shards10_500': array([267.634635]),\n",
       " 'PGD_2_SISA-DK_shards10_1000': array([253.7464]),\n",
       " 'PGD_2_SISA-DK_shards10_1500': array([501.67434543]),\n",
       " 'PGD_2_SISA-DK_shards10_2000': array([488.63563452]),\n",
       " 'PGD_2_SISA-DK_shards10_2500': array([712.53453463]),\n",
       " 'PGD_2_SISA-DK_shards10_3000': array([917.34635634]),\n",
       " 'PGD_2_SISA-DK_shards20_500': array([127.45353522]),\n",
       " 'PGD_2_SISA-DK_shards20_1000': array([117.6345632]),\n",
       " 'PGD_2_SISA-DK_shards20_1500': array([179.534535]),\n",
       " 'PGD_2_SISA-DK_shards20_2000': array([256.463453]),\n",
       " 'PGD_2_SISA-DK_shards20_2500': array([297.235235]),\n",
       " 'PGD_2_SISA-DK_shards20_3000': array([438.53453466]),\n",
       " 'PGD_2_MUter500': array([43.85744047]),\n",
       " 'PGD_2_MUter1000': array([55.8829248]),\n",
       " 'PGD_2_MUter1500': array([66.01313066]),\n",
       " 'PGD_2_MUter2000': array([77.30215716]),\n",
       " 'PGD_2_MUter2500': array([87.61671424]),\n",
       " 'PGD_2_MUter3000': array([99.32522798]),\n",
       " 'PGD_2_FMUter500': array([13.05112195]),\n",
       " 'PGD_2_FMUter1000': array([24.61391521]),\n",
       " 'PGD_2_FMUter1500': array([34.82374954]),\n",
       " 'PGD_2_FMUter2000': array([45.72249627]),\n",
       " 'PGD_2_FMUter2500': array([56.71987653]),\n",
       " 'PGD_2_FMUter3000': array([71.07160258])}"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recorder4.time_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "recorder5 = Recorder(args)\n",
    "recorder5.load(time_method_list=arr, times=13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'PGD_2_SISA-DK_shards5_500': array([988.3425334]),\n",
       " 'PGD_2_SISA-DK_shards5_1000': array([972.563432]),\n",
       " 'PGD_2_SISA-DK_shards5_1500': array([983.53482579]),\n",
       " 'PGD_2_SISA-DK_shards5_2000': array([1054.53103]),\n",
       " 'PGD_2_SISA-DK_shards5_2500': array([1844.64554334]),\n",
       " 'PGD_2_SISA-DK_shards5_3000': array([1991.21756323]),\n",
       " 'PGD_2_SISA-DK_shards10_500': array([241.002831]),\n",
       " 'PGD_2_SISA-DK_shards10_1000': array([281.7523534]),\n",
       " 'PGD_2_SISA-DK_shards10_1500': array([519.0579104]),\n",
       " 'PGD_2_SISA-DK_shards10_2000': array([517.45423902]),\n",
       " 'PGD_2_SISA-DK_shards10_2500': array([745.9481134]),\n",
       " 'PGD_2_SISA-DK_shards10_3000': array([1025.67498423]),\n",
       " 'PGD_2_SISA-DK_shards20_500': array([131.9637492]),\n",
       " 'PGD_2_SISA-DK_shards20_1000': array([122.96389223]),\n",
       " 'PGD_2_SISA-DK_shards20_1500': array([168.84682635]),\n",
       " 'PGD_2_SISA-DK_shards20_2000': array([243.84567282]),\n",
       " 'PGD_2_SISA-DK_shards20_2500': array([301.94728378]),\n",
       " 'PGD_2_SISA-DK_shards20_3000': array([415.86937296]),\n",
       " 'PGD_2_MUter500': array([44.14869142]),\n",
       " 'PGD_2_MUter1000': array([54.74314094]),\n",
       " 'PGD_2_MUter1500': array([66.54231095]),\n",
       " 'PGD_2_MUter2000': array([77.44492626]),\n",
       " 'PGD_2_MUter2500': array([88.66816974]),\n",
       " 'PGD_2_MUter3000': array([99.68387151]),\n",
       " 'PGD_2_FMUter500': array([12.34689069]),\n",
       " 'PGD_2_FMUter1000': array([24.07012105]),\n",
       " 'PGD_2_FMUter1500': array([34.501266]),\n",
       " 'PGD_2_FMUter2000': array([46.92237067]),\n",
       " 'PGD_2_FMUter2500': array([63.54910111]),\n",
       " 'PGD_2_FMUter3000': array([49.68954325])}"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recorder5.time_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Recorder3.clean_acc_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Recorder3.perturbed_acc_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Recorder3.distance_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# arr = [5396.41852498,  5082.7408824 ,  4856.18920207,  4574.80710912]\n",
    "# for item in arr:\n",
    "#     recorder1.metrics_time_record(method='SISA', time=item)\n",
    "# recorder1.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder.time_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder1.time_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder2.time_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder.distance_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder.perturbed_acc_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder.clean_acc_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder1.distance_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder1.clean_acc_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder1.perturbed_acc_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder1.time_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder2.distance_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder2.clean_acc_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder2.perturbed_acc_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder2.time_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from utils import acc_abs\n",
    "# acc_abs(recorder1)\n",
    "# acc_abs(recorder2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder1.clean_acc_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder1.perturbed_acc_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder2.clean_acc_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# recorder2.perturbed_acc_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from utils import Transform_to_dataframe\n",
    "# import seaborn as sns\n",
    "\n",
    "# df1_clean_acc = Transform_to_dataframe(recorder1.clean_acc_dict, [1, 200, 500, 1000, 2000, 4000], args)\n",
    "# df2_clean_acc = Transform_to_dataframe(recorder2.clean_acc_dict, [1, 200, 500, 1000, 2000, 4000], args)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pandas as pd\n",
    "\n",
    "# df_acc = pd.concat((df1_clean_acc, df2_clean_acc), ignore_index=True)\n",
    "# df_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from utils import bar_plot\n",
    "# bar_plot(df_acc, 'Acc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from utils import Transform_to_dataframe\n",
    "# import seaborn as sns\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # [1, 200, 500, 1000, 2000, 4000]\n",
    "# df = Transform_to_dataframe(recorder.distance_dict, [1, 200, 500, 1000, 2000, 4000], args)\n",
    "# df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df1 = Transform_to_dataframe(recorder1.distance_dict, [1, 200, 500, 1000, 2000, 4000], args)\n",
    "# df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df2 = Transform_to_dataframe(recorder2.distance_dict, [1, 200, 500, 1000, 2000, 4000], args)\n",
    "# df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pandas as pd\n",
    "\n",
    "# df_sum = pd.concat((df, df1, df2), ignore_index=True)\n",
    "# df_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from utils import line_plot\n",
    "\n",
    "# line_plot(df_sum, metrics='Distance')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import copy\n",
    "# df_copy =  copy.deepcopy(df)\n",
    "# df_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pandas as pd\n",
    "\n",
    "# sums = pd.concat((df, df_copy), ignore_index=True)\n",
    "# sums"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sns.set_style('darkgrid')\n",
    "# ax = sns.lineplot(x='index', y='value', data=df, hue='method', markers=['o' for i in range(7)], style='method', \n",
    "# dashes=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sns.set_style('darkgrid')\n",
    "# bx = sns.barplot(data=df, x='method', y='value', hue='index')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('xms_torch')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "419209cf9e31416c5930fc7553ba605783d1608c64e9bc9d407f3ec9a261c809"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
